import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;


public class StopWordsNormalizer implements Normalizer {
	private final Set<String> stop;
	
	public StopWordsNormalizer() throws IOException {
		stop = new HashSet<String>();
		BufferedReader r = new BufferedReader(new FileReader("stoplist.txt"));		
		String str;
		while((str = r.readLine()) != null) {
			stop.add(str);
		}
		r.close();
	}
	
	@Override
	public Set<String> getTerms(List<String> tokens) throws IOException {
		Set<String> terms = new HashSet<String>();
		
		for(String t : tokens) {
			if (!stop.contains(t)) {
				terms.add(t);
			}
		}
		
		return terms;
	}

}
